#
# my $t = TWSE->new( verbose => 1, refine => 0 );
# $t->get_info( [DataSource object] );
#
#
#

package TWSE;
use strict;
use warnings;
use Retriever;
use utf8;

sub new {
    my $invocant = shift;
    my $class = ref($invocant) || $invocant;
    my $self = {};
    bless ($self, $class);
    $self->_init(@_);
    return $self;
}

sub _init {
    my $self = shift;
    my %param = @_;
    $self->{_verbose} = $param{verbose} || 1;
    $self->{_refine} = $param{refine} || 0;
}

sub get_info {
    my $self = shift;
    my $ds = $_[0];
    if ( ! defined $ds or ref $ds ne "DataSource" ){
        die "DataSource object required!\n";
    }
    my @info;
    my $r = new Retriever;
    foreach my $req ( @{$ds->reqs} ){
        $r->retrieve( $req );
        my $info = $r->extract_table( 
            attribs => $ds->attr, 
            decode => $ds->code
        );
        push @info, @$info;
        print "downloading ".$ds->desc."... (".(scalar @$info)." recoreds retrieved)\n" if $self->{_verbose};
        sleep 3;
    }

    # get column descriptors.
    my $ds_cols = $ds->columns;

    # refine retrieved information for database storage
    if ( $self->{_refine} ){
    foreach my $row ( @info ){
        foreach my $col_name ( keys %$row ){
            if ( defined $ds_cols->{$col_name} ){
                my $new_cname = $ds_cols->{$col_name}->{col_name};
                $row->{$new_cname} = delete $row->{$col_name};
                if ( $ds_cols->{$col_name}->{type} eq "numerical" ){
                    $row->{$new_cname} =~ s/[^\d^\.^+^\-^\w]//g;
                } elsif ( $ds_cols->{$col_name}->{type} eq "date" ){
                    if ( $row->{$new_cname} =~ /(\d+)\/(\d+)\/(\d+)/ ){
                        $row->{$new_cname} = $1 + 1911 ."-".$2."-".$3;
                    }
                }
            } else {
                delete $row->{$col_name};
            }
        }
    }
    }

    return @info;
}

1;
